package main

import (
	"gxgeek.com/gxgeek/learngo/crawler/engine"
	"gxgeek.com/gxgeek/learngo/crawler/zhenai/parser"
	"gxgeek.com/gxgeek/learngo/crawler/scheduler"
	"gxgeek.com/gxgeek/learngo/crawler/persist"
	"gxgeek.com/gxgeek/learngo/crawler/config"
)

const ZHENG_HUN_URL  = "http://www.zhenai.com/zhenghun"
const PROFILE_INDEX  = "dating_profile"

func main() {
	saver, err := persist.ItemSaver(PROFILE_INDEX)
	if err!=nil{
		panic(err)
	}
	concurrentEngine := engine.ConcurrentEngine{
		Scheduler: &scheduler.QueuedScheduler{},
		WorkerCount:100,
		ItemChan:saver,
		RequestProcessor: engine.Worker,
	}
	concurrentEngine.Run(engine.Request{
		Url:ZHENG_HUN_URL,
		Parser:engine.NewFuncParser(parser.ParseCityList,config.ParseCity),
	})

	//concurrentEngine := engine.ConcurrentEngine{
	//	Scheduler: &scheduler.QueuedScheduler{},
	//	WorkerCount:100,
	//	ItemChan:persist.ItemSaver(),
	//}
	//concurrentEngine.Run(engine.Requests{
	//	Url:"http://www.zhenai.com/zhenghun/shanghai",
	//	ParserFunc:parser.ParseCity,
	//})

	//e := engine.ConcurrentEngine{
	//	Scheduler: &scheduler.SimpleScheduler{},
	//	WorkerCount:100,
	//}
	//e.Run(engine.Requests{
	//	Url:ZHENG_HUN_URL,
	//	ParserFunc:parser.ParseCityList,
	//})


}
